def objective_xgboost(trial):
# Fold and seed
train = pd.read_csv("../data/train_sanitized.csv")
X_train, y_train = train.drop(['interest_rate'], axis=1), train.interest_rate.to_numpy()
folds = 5
seed = 1227
# Parameters
search_space = {
# Booster parameters
'booster_params': {
'booster': 'gbtree',
'objective': 'reg:squarederror',
'eval_metric': 'rmse', # Use RMSE for evaluation metric on train and validation sets
'learning_rate': trial.suggest_float(name='learning_rate', low=0.001, high=0.5), # Range: [0, 1], larger eta shrinks the feature weights more to make the boosting process more conservative, i.e., fewer trees (regularizer)
'gamma': trial.suggest_int('gamma', 0, 20), # Range: [0, inf], the larger the more conservative the algorithm (regularizer)
'max_delta_step': trial.suggest_int('max_delta_step', 1, 10), # Range: [0, inf], values from 1-10 might help control the update for imbalanced data (regularizer)
'lambda': trial.suggest_categorical('lambda', [10, 100, 500]), # Range: [0, inf], L2 regularization term on weights, the larger the more conservative the algorithm (regularizer)
'alpha': trial.suggest_categorical('alpha', [10, 100, 500]), # Range: [0, inf], L1 regularization term on weights, the larger the more conservative the algorithm (regularizer)
'colsample_bylevel': trial.suggest_categorical('colsample_bylevel', np.linspace(0.3, 1, 6).tolist()),
'colsample_bynode': trial.suggest_categorical('colsample_bynode', np.linspace(0.3, 1, 6).tolist()),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', np.linspace(0.3, 1, 6).tolist()), # Range: (0, 1], subsample ratio of columns when constructing each tree, the smaller the more conservative the algorithm (regularizer)
'subsample': trial.suggest_categorical('subsample', np.linspace(0.3, 1, 6).tolist()), # Range: (0, 1], subsample ratio of the training instances every boosting iteration, the smaller the more conservative the algorithm (regularizer)
'sampling_method': 'gradient_based', # Only supported for 'gpu_hist'
'max_depth': trial.suggest_categorical('max_depth', np.arange(3, 12, dtype=np.int16).tolist()), # Range: [0, inf], deep trees boost predictive power but are more likely to overfit (bias reducer)
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
},
# Non-booster parameters
'num_boost_round': trial.suggest_int('num_boost_round', low=500, high=2000, step=100), # Range: [0, inf], number of boosting iterations, the larger the more likely to overfit (bias reducer)
}
# K-fold cross validation
kf = KFold(n_splits=folds, shuffle=True, random_state=rs)
rmse_scores = np.empty(folds)
for fold, (train_indx, val_indx) in enumerate(kf.split(X_train, y_train)):
# Train and validation sets
fold_X_train, fold_y_train = X_train.iloc[train_indx], y_train[train_indx]
fold_X_val, fold_y_val = X_train.iloc[val_indx], y_train[val_indx]
# Preprocessing using a fresh copy of the pipeline for every fold to prevent leakage
preprocessor = joblib.load('../output/preprocessors/xgboost_preprocessor.joblib')
print(f'Start processing fold {fold + 1}...')
fold_X_train = preprocessor.fit_transform(fold_X_train, fold_y_train)
fold_X_val = preprocessor.transform(fold_X_val)
# Data for modeling
feature_names = fold_X_train.columns.tolist()
dtrain = xgb.DMatrix(data=fold_X_train, label=fold_y_train, feature_names=feature_names)
dvalid = xgb.DMatrix(data=fold_X_val, label=fold_y_val, feature_names=feature_names)
# Model
model = xgb.train(
params=search_space['booster_params'],
dtrain=dtrain,
num_boost_round=search_space['num_boost_round'],
early_stopping_rounds=200,
evals=[(dtrain, 'train'), (dvalid, 'validate')],
verbose_eval=200 # Print eval every 200 boosting rounds
)
# Out-of-fold prediction
print(f'Predicting for fold {fold + 1}...')
oof_pred = model.predict(data=dvalid)
rmse_scores[fold] = mean_squared_error(fold_y_val, oof_pred, squared=False) # Use RMSE
# Average across 5 folds
mean_rmse = np.mean(rmse_scores)
return mean_rmse
